Josh Dillon, Last Revised January 2022
This notebook examines an individual antenna's performance over a whole season. This notebook parses information from each nightly rtp_summarynotebook (as saved to .csvs) and builds a table describing antenna performance. It also reproduces per-antenna plots from each auto_metrics notebook pertinent to the specific antenna.
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder
# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')
antenna = "114" csv_folder = "/home/obs/src/H6C_Notebooks/_rtp_summary_" auto_metrics_folder = "/home/obs/src/H6C_Notebooks/auto_metrics_inspect"
display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna
# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')
Found 75 csvs in /home/obs/src/H6C_Notebooks/_rtp_summary_ Found 73 auto_metrics notebooks in /home/obs/src/H6C_Notebooks/auto_metrics_inspect
# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0
def jd_to_summary_url(jd):
return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'
def jd_to_auto_metrics_url(jd):
return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'
this_antenna = None
jds = []
# parse information about antennas and nodes
for csv in csvs:
df = pd.read_csv(csv)
for n in range(len(df)):
# Add this day to the antenna
row = df.loc[n]
if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
else:
antnum = int(row['Ant'])
if antnum != int(antenna):
continue
if np.issubdtype(type(row['Node']), np.integer):
row['Node'] = str(row['Node'])
if type(row['Node']) == str and row['Node'].isnumeric():
row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
if this_antenna is None:
this_antenna = Antenna(row['Ant'], row['Node'])
jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
jds.append(jd)
this_antenna.add_day(jd, row)
break
# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]
df = pd.DataFrame(to_show)
# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
df[col] = bar_cols[col]
z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
df[col] = z_score_cols[col]
ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
df[col] = ant_metrics_cols[col]
redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]
for col in redcal_cols:
df[col] = redcal_cols[col]
# style dataframe
table = df.style.hide_index()\
.applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
.background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
.background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
.background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
.background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
.applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
.applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
.applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
.applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
.bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
.format({col: '{:,.4f}'.format for col in z_score_cols}) \
.format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
.format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
.set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])
This table reproduces each night's row for this antenna from the RTP Summary notebooks. For more info on the columns, see those notebooks, linked in the JD column.
display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))
| JDs | A Priori Status | Auto Metrics Flags | Dead Fraction in Ant Metrics (Jee) | Dead Fraction in Ant Metrics (Jnn) | Crossed Fraction in Ant Metrics | Flag Fraction Before Redcal | Flagged By Redcal chi^2 Fraction | ee Shape Modified Z-Score | nn Shape Modified Z-Score | ee Power Modified Z-Score | nn Power Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Discontinuties Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | Average Dead Ant Metric (Jee) | Average Dead Ant Metric (Jnn) | Average Crossed Ant Metric | Median chi^2 Per Antenna (Jee) | Median chi^2 Per Antenna (Jnn) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2459891 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 5.525733 | 0.515977 | 0.856346 | -0.325425 | 4.653433 | -1.274421 | 1.814205 | -0.657505 | 0.5256 | 0.6458 | 0.4236 | nan | nan |
| 2459890 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 5.783251 | 0.443875 | 0.868106 | -0.235852 | 2.248424 | -1.945527 | 2.693898 | -0.501898 | 0.5356 | 0.6423 | 0.4176 | nan | nan |
| 2459889 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 6.588265 | 0.604294 | 0.514425 | -0.433540 | 4.866827 | -2.778196 | 0.957388 | -0.874844 | 0.5416 | 0.6487 | 0.4155 | nan | nan |
| 2459888 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 5.256151 | 0.506127 | 0.971661 | -0.127071 | 4.150579 | -1.548719 | 0.897547 | -1.227247 | 0.5707 | 0.6716 | 0.4071 | nan | nan |
| 2459887 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 5.739977 | 0.461515 | 1.141191 | -0.844700 | 2.283931 | -1.394490 | 1.156610 | -0.360069 | 0.5467 | 0.6495 | 0.4198 | nan | nan |
| 2459886 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 5.934321 | 0.578338 | 1.065312 | -0.173328 | 0.434402 | -1.473736 | 0.714633 | -1.385608 | 0.6437 | 0.7079 | 0.3585 | nan | nan |
| 2459885 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 9.976955 | 2.621715 | 3.962356 | 20.777696 | 5.834468 | 3.655464 | 1.287150 | 3.421329 | 0.6137 | 0.6901 | 0.3761 | nan | nan |
| 2459884 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 7.362672 | 3.854452 | 2.319198 | 3.912474 | 2.176820 | 4.069985 | 0.593743 | -2.861842 | 0.4895 | 0.6444 | 0.4335 | nan | nan |
| 2459883 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 10.861089 | 6.488178 | 11.885134 | 37.924228 | 3.810179 | 6.647768 | 2.271415 | -5.013536 | 0.4930 | 0.6556 | 0.4311 | nan | nan |
| 2459882 | not_connected | 100.00% | 100.00% | 100.00% | 0.00% | - | - | 27.338095 | 25.018311 | 81.495557 | 70.271359 | 19.122523 | 20.006650 | 221.294779 | 191.664211 | 0.0257 | 0.0291 | 0.0025 | nan | nan |
| 2459881 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 10.324114 | 6.658155 | 14.440665 | 49.800331 | 12.061897 | 17.582496 | 3.370948 | 1.455455 | 0.5524 | 0.7090 | 0.3668 | nan | nan |
| 2459880 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 12.720885 | 7.267264 | 12.696190 | 40.425812 | 4.318995 | 5.782322 | 2.003176 | -2.926219 | 0.4831 | 0.6537 | 0.4397 | nan | nan |
| 2459879 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 6.781476 | 2.721450 | 2.517459 | 1.209308 | 1.579212 | -0.756720 | 0.882033 | -3.367808 | 0.4512 | 0.6455 | 0.4563 | nan | nan |
| 2459878 | not_connected | 100.00% | 0.00% | 0.00% | 0.00% | - | - | 11.691610 | 7.096530 | 15.610369 | 49.431211 | 7.507741 | 10.103976 | 3.966605 | -5.059550 | 0.4747 | 0.6573 | 0.4456 | nan | nan |
auto_metrics notebooks.¶htmls_to_display = []
for am_html in auto_metric_htmls:
html_to_display = ''
# read html into a list of lines
with open(am_html) as f:
lines = f.readlines()
# find section with this antenna's metric plots and add to html_to_display
jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
try:
section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
except ValueError:
continue
html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
for line in lines[section_start_line + 1:]:
html_to_display += line
if '<hr' in line:
htmls_to_display.append(html_to_display)
break
These figures are reproduced from auto_metrics notebooks. For more info on the specific plots and metrics, see those notebooks (linked at the JD). The most recent 100 days (at most) are shown.
for i, html_to_display in enumerate(htmls_to_display):
if i == 100:
break
display(HTML(html_to_display))
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | ee Shape Modified Z-Score | nn Shape Modified Z-Score | ee Power Modified Z-Score | nn Power Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Discontinuties Modified Z-Score | nn Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 5.525733 | 5.525733 | 0.515977 | 0.856346 | -0.325425 | 4.653433 | -1.274421 | 1.814205 | -0.657505 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 5.783251 | 0.443875 | 5.783251 | -0.235852 | 0.868106 | -1.945527 | 2.248424 | -0.501898 | 2.693898 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | ee Shape Modified Z-Score | nn Shape Modified Z-Score | ee Power Modified Z-Score | nn Power Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Discontinuties Modified Z-Score | nn Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 6.588265 | 6.588265 | 0.604294 | 0.514425 | -0.433540 | 4.866827 | -2.778196 | 0.957388 | -0.874844 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 5.256151 | 0.506127 | 5.256151 | -0.127071 | 0.971661 | -1.548719 | 4.150579 | -1.227247 | 0.897547 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 5.739977 | 0.461515 | 5.739977 | -0.844700 | 1.141191 | -1.394490 | 2.283931 | -0.360069 | 1.156610 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | ee Shape Modified Z-Score | nn Shape Modified Z-Score | ee Power Modified Z-Score | nn Power Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Discontinuties Modified Z-Score | nn Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 5.934321 | 5.934321 | 0.578338 | 1.065312 | -0.173328 | 0.434402 | -1.473736 | 0.714633 | -1.385608 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | nn Power | 20.777696 | 2.621715 | 9.976955 | 20.777696 | 3.962356 | 3.655464 | 5.834468 | 3.421329 | 1.287150 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 7.362672 | 3.854452 | 7.362672 | 3.912474 | 2.319198 | 4.069985 | 2.176820 | -2.861842 | 0.593743 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | nn Power | 37.924228 | 6.488178 | 10.861089 | 37.924228 | 11.885134 | 6.647768 | 3.810179 | -5.013536 | 2.271415 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Temporal Discontinuties | 221.294779 | 25.018311 | 27.338095 | 70.271359 | 81.495557 | 20.006650 | 19.122523 | 191.664211 | 221.294779 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | nn Power | 49.800331 | 6.658155 | 10.324114 | 49.800331 | 14.440665 | 17.582496 | 12.061897 | 1.455455 | 3.370948 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | nn Power | 40.425812 | 7.267264 | 12.720885 | 40.425812 | 12.696190 | 5.782322 | 4.318995 | -2.926219 | 2.003176 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | ee Shape | 6.781476 | 2.721450 | 6.781476 | 1.209308 | 2.517459 | -0.756720 | 1.579212 | -3.367808 | 0.882033 |
| Ant | Node | A Priori Status | Worst Metric | Worst Modified Z-Score | nn Shape Modified Z-Score | ee Shape Modified Z-Score | nn Power Modified Z-Score | ee Power Modified Z-Score | nn Temporal Variability Modified Z-Score | ee Temporal Variability Modified Z-Score | nn Temporal Discontinuties Modified Z-Score | ee Temporal Discontinuties Modified Z-Score |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 114 | N11 | not_connected | nn Power | 49.431211 | 7.096530 | 11.691610 | 49.431211 | 15.610369 | 10.103976 | 7.507741 | -5.059550 | 3.966605 |